import requests
from jsonpath import jsonpath
import re,json
from pprint import pprint
import time
import csv


class Spider():
    def __init__(self):
        self.headers = {
        'authority': 'mapi.vip.com',
        'accept': '*/*',
        'accept-language': 'zh-CN,zh;q=0.9',
        'cookie': 'vip_cps_cuid=CU1703946155093f91b22c68d55b7591; vip_cps_cid=1703946155095_f8aebf721aa4d69f55487762e3ca4c21; cps_share=cps_share; PAPVisitorId=58460c7a22e31f6b4acb2a1ed741f921; vip_new_old_user=1; vip_city_name=%E5%B9%BF%E5%B7%9E%E5%B8%82; mars_cid=1703946163504_4eebec221de3364e0da3bbe4a2182454; VipUINFO=luc%3Aa%7Csuc%3Aa%7Cbct%3Ac_new%7Chct%3Ac_new%7Cbdts%3A0%7Cbcts%3A0%7Ckfts%3A0%7Cc10%3A0%7Crcabt%3A0%7Cp2%3A0%7Cp3%3A1%7Cp4%3A0%7Cp5%3A0%7Cul%3A3105; mars_pid=0; vip_address=%257B%2522pname%2522%253A%2522%255Cu5b89%255Cu5fbd%255Cu7701%2522%252C%2522pid%2522%253A%2522103104%2522%252C%2522cname%2522%253A%2522%255Cu5e7f%255Cu5dde%255Cu5e02%2522%252C%2522cid%2522%253A%2522103104114%2522%257D; vip_province=103104; vip_province_name=%E5%AE%89%E5%BE%BD%E7%9C%81; vip_city_code=103104114; vip_wh=VIP_HZ; vip_ipver=31; user_class=a; cps=adp%3Antq8exyc%3A%40_%401704022867205%3Amig_code%3A4f6b50bf15bfa39639d85f5f1e15b10f%3Aac014miuvl0000b5sq8crnthcjdwurb0; mars_sid=ef0bd4aed17dd0eb261cda2a1a73e9d8; visit_id=43601963E9569AF31FBF7F1561D38FC9; vipshop_passport_src=https%3A%2F%2Fcategory.vip.com%2Fsuggest.php%3Fkeyword%3D%25E5%258F%25A3%25E7%25BA%25A2%26ff%3D235%7C12%7C1%7C1; vip_tracker_source_from=; vip_access_times=%7B%22list%22%3A0%2C%22detail%22%3A2%7D; pg_session_no=6; VipDFT=1',
        'referer': 'https://category.vip.com/',
        'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"',
        'sec-fetch-dest': 'script',
        'sec-fetch-mode': 'no-cors',
        'sec-fetch-site': 'same-site',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
    }
        self.home_url = 'https://mapi.vip.com/vips-mobile/rest/shopping/pc/product/module/list/v2'
        self.num = 1

    def praseHomedata(self,product):
        try:
            self.sub_data = {
                'callback': ' getMerchandiseDroplets1',
                'app_name': ' shop_pc',
                'app_version': ' 4.0',
                'warehouse': ' VIP_HZ',
                'fdc_area_id': ' 103104114',
                'client': ' pc',
                'mobile_platform': ' 1',
                'province_id': ' 103104',
                'api_key': ' 70f71280d5d547b2a7bb370a529aeea1',
                'user_id': ' ',
                'mars_cid': ' 1703946163504_4eebec221de3364e0da3bbe4a2182454',
                'wap_consumer': ' a',
                'productIds': product,
                'scene': ' search',
                'standby_id': ' nature',
                'extParams': ' {"stdSizeVids":"","preheatTipsVer":"3","couponVer":"v2","exclusivePrice":"1","iconSpec":"2x","ic2label":1,"superHot":1,"bigBrand":"1"}',
                'context': ' ',
                '_': '1704027272052'
            }

            time.sleep(1)
            response = requests.get(self.home_url,headers=self.headers,params=self.sub_data).text
            json_data = response.split('getMerchandiseDroplets1(')[1]
            data = re.sub('\)', '', json_data)
            json_msg = json.loads(data)

            for i in json_msg['data']['products']:
                attrs = jsonpath(i,'$..value')[0]
                dict = {
                    "标题" : i['title'],
                    "品牌" : i['brandShowName'],
                    "原价" : i['price']['marketPrice'],
                    "折扣" : i['price']['saleDiscount'],
                    "现价" : i['price']['salePrice'],
                    "属性" : attrs
                }
                spuid=i['spuId']
                dict=self.getcomment(dict,spuid)
                pprint(dict)
                print('===========================================')
                csv_write.writerow(dict)
        except:
            print("爬取完毕")

    def getcomment(self,dict,spuid):
       self.comment_url='https://mapi.vip.com/vips-mobile/rest/ugc/reputation/getSpuIdNlpKeywordV2_for_pc'
       headers_= {
          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.289 Safari/537.36',
          'Referer':'https://detail.vip.com/',
          'Cookie':'cps=adp%3Ag1071nr0%3A%3A%3A%3A: vip first visilr=1; vip address-%257B%2522pid%2522%253A%2522104101%2522%252C%2522cd%2522%.253A%2522104101102.252%.252C%2522pname%2522%.253A%2522%.255Cub3%255Cu5357%255Cu7701%2522%,252C%.2522name%252%253A%252%255Cu5100%255Cu5c01%255Cu5e02%2522%257D;vip province=104101: vip province name-%E6%B2%B3%E5%8D%97%E7%9C%81; vip city name-%E5%BC%80%E5%B0%81%E5%B8%82: vip city code=104101102: vip wh=VIP HZ: vip pver-31; user class-a; mars sid-f59dic6694b384e51007ta4e7a9d864: PHPSESSID-1ndikirkaprn0eavenmdt32v7: mars pid-0: visit id-8482B9A56BDOEDOF7428399DD6B79874: VipuNFO=luc%3Aa%7CSuc%3Aa%7Cbct%3Ac new%7Chct%3Ac new%7Cbdts%3A0%7Cbcts%3A0%7Ckhts%3A0%7Cc10%3A0%7Crcabi%3A0%7Co2%3A0%7Cp3%3A1%7Cp4%3A0%7Cp5%3A1%7Cu%3A3105: vip tracker source from=; pg session no-16: mars cid-161198562068 86046d3ae583d23339a1a310c41f271f'
}
       comment_data={
           'callback': 'getSpuIdNlpKeywordCb',
           'app_name': 'shop_pc',
           'app_version': '4.0',
           'warehouse': 'VIP_CD',
           'fdc_area_id': '106101101',
           'client': 'pc',
           'mobile_platform': '1',
           'province_id': '106101',
           'api_key': '70f71280d5d547b2a7bb370a529aeea1',
           'user_id': '',
           'mars_cid': '1705231820539_08c25e658e622d85cd66b33d123dcb52',
           'wap_consumer': 'a',
           'showTagOther': '1',
           'spuId': str(spuid),
           '_':'1705414834195'
       }
       response=requests.get(url=self.comment_url,headers=headers_,params=comment_data).text
       json_data= response.split('getSpuIdNlpKeywordCb(')[1]
       print(json_data)
       data= re.sub('\)', '', json_data)
       json_msg= json.loads(data)
       print(json_msg)
       dict["满意度"]=json_msg['data']['satisfaction']
       print(json_msg['data']['satisfaction'])
       dict["评论条数"]=json_msg['data']['nlpKeyWordList'][0]['asKeyWordCount']
       print(json_msg['data']['nlpKeyWordList'][0]['asKeyWordCount'])
       return dict

    def praseSubpage(self):
        for page in range(0, 2400, 120):
            print(f"正在爬取第{self.num}页")
            # 商品详情链接

            # 产品编码数据
            sub_url = f"https://mapi.vip.com/vips-mobile/rest/shopping/pc/search/product/rank?callback=getMerchandiseIds&app_name=shop_pc&app_version=4.0&warehouse=VIP_CD&fdc_area_id=106101101&client=pc&mobile_platform=1&province_id=106101&api_key=70f71280d5d547b2a7bb370a529aeea1&user_id=&mars_cid=1705231820539_08c25e658e622d85cd66b33d123dcb52&wap_consumer=a&standby_id=nature&keyword=%E5%8C%96%E5%A6%86%E5%93%81&lv3CatIds=&lv2CatIds=&lv1CatIds=&brandStoreSns=&props=&priceMin=&priceMax=&vipService=&sort=0&pageOffset={page}&channelId=1&gPlatform=PC&batchSize=120&_=1705504495043"
            # 请求链接得到字典
            response = requests.get(sub_url, headers=self.headers).text
            # 获得产品编码
            data = re.findall('{"pid":"(.*?)"}', response)
            # 合并字符串

            product1 = ','.join(data[0:50])
            print(product1)
            product2 = ','.join(data[50:100])
            product3 = ','.join(data[100:120])
            self.praseHomedata(product1)
            self.praseHomedata(product2)
            self.praseHomedata(product3)
            self.num += 1



if __name__ == '__main__':
    with open('唯品会美妆数据附加3.csv', 'a', encoding='utf-8', newline='')as f:
        csv_write = csv.DictWriter(f, fieldnames=["标题", "品牌", "原价", "折扣", "现价", "属性","满意度","评论条数"])
        csv_write.writeheader()
        spider = Spider()
        spider.praseSubpage()

